

<!DOCTYPE html>
<html lang="zh-CN" data-default-color-scheme=auto>



<head>
  <meta charset="UTF-8">
  <link rel="apple-touch-icon" sizes="76x76" href="/img/favicon.png">
  <link rel="icon" href="/img/favicon.png">
  <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=5.0, shrink-to-fit=no">
  <meta http-equiv="x-ua-compatible" content="ie=edge">
  
  <meta name="theme-color" content="#2f4154">
  <meta name="author" content="KD">
  <meta name="keywords" content="">
  
    <meta name="description" content="requests模块Response对象的属性     属性 说明     r.status_code HTTP请求的返回状态，200表示连接成功，404表示失败   r.text HTTP响应内容的字符串形式，即url对应的页面内容   r.encoding 从HTTP header中猜测的响应内容编码方式   r.apparent_encoding 从内容中分析出的响应编码方式（备选编码方式）">
<meta property="og:type" content="article">
<meta property="og:title" content="MOOC爬虫">
<meta property="og:url" content="https://kdlong.gitee.io/posts/2021/04-227548fa7ce917.html">
<meta property="og:site_name" content="KD的博客">
<meta property="og:description" content="requests模块Response对象的属性     属性 说明     r.status_code HTTP请求的返回状态，200表示连接成功，404表示失败   r.text HTTP响应内容的字符串形式，即url对应的页面内容   r.encoding 从HTTP header中猜测的响应内容编码方式   r.apparent_encoding 从内容中分析出的响应编码方式（备选编码方式）">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://api.ixiaowai.cn/mcapi/mcapi.php?09">
<meta property="article:published_time" content="2021-04-22T01:12:39.861Z">
<meta property="article:modified_time" content="2022-05-18T13:05:15.569Z">
<meta property="article:author" content="KD">
<meta property="article:tag" content="Python">
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:image" content="https://api.ixiaowai.cn/mcapi/mcapi.php?09">
  
  
    <meta name="referrer" content="no-referrer-when-downgrade">
  
  
  <title>MOOC爬虫 - KD的博客</title>

  <link  rel="stylesheet" href="https://lib.baomitu.com/twitter-bootstrap/4.6.1/css/bootstrap.min.css" />



  <link  rel="stylesheet" href="https://lib.baomitu.com/github-markdown-css/4.0.0/github-markdown.min.css" />

  <link  rel="stylesheet" href="https://lib.baomitu.com/hint.css/2.7.0/hint.min.css" />

  <link  rel="stylesheet" href="https://lib.baomitu.com/fancybox/3.5.7/jquery.fancybox.min.css" />



<!-- 主题依赖的图标库，不要自行修改 -->
<!-- Do not modify the link that theme dependent icons -->

<link rel="stylesheet" href="//at.alicdn.com/t/font_1749284_hj8rtnfg7um.css">



<link rel="stylesheet" href="//at.alicdn.com/t/font_1736178_lbnruvf0jn.css">


<link  rel="stylesheet" href="/css/main.css" />


  <link id="highlight-css" rel="stylesheet" href="/css/highlight.css" />
  
    <link id="highlight-css-dark" rel="stylesheet" href="/css/highlight-dark.css" />
  




  <script id="fluid-configs">
    var Fluid = window.Fluid || {};
    Fluid.ctx = Object.assign({}, Fluid.ctx)
    var CONFIG = {"hostname":"kdlong.gitee.io","root":"/","version":"1.9.0","typing":{"enable":true,"typeSpeed":70,"cursorChar":"|","loop":false,"scope":[]},"anchorjs":{"enable":true,"element":"h1,h2,h3,h4,h5,h6","placement":"right","visible":"hover","icon":"§"},"progressbar":{"enable":true,"height_px":3,"color":"#29d","options":{"showSpinner":false,"trickleSpeed":100}},"code_language":{"enable":true,"default":"TEXT"},"copy_btn":true,"image_caption":{"enable":true},"image_zoom":{"enable":true,"img_url_replace":["",""]},"toc":{"enable":true,"placement":"right","headingSelector":"h1,h2,h3,h4,h5,h6","collapseDepth":2},"lazyload":{"enable":true,"loading_img":"/img/loading.gif","onlypost":false,"offset_factor":2},"web_analytics":{"enable":false,"follow_dnt":true,"baidu":null,"google":null,"gtag":null,"tencent":{"sid":null,"cid":null},"woyaola":null,"cnzz":null,"leancloud":{"app_id":null,"app_key":null,"server_url":null,"path":"window.location.pathname","ignore_local":false}},"search_path":"/local-search.xml"};

    if (CONFIG.web_analytics.follow_dnt) {
      var dntVal = navigator.doNotTrack || window.doNotTrack || navigator.msDoNotTrack;
      Fluid.ctx.dnt = dntVal && (dntVal.startsWith('1') || dntVal.startsWith('yes') || dntVal.startsWith('on'));
    }
  </script>
  <script  src="/js/utils.js" ></script>
  <script  src="/js/color-schema.js" ></script>

  
<meta name="generator" content="Hexo 5.4.0"><link rel="alternate" href="/atom.xml" title="KD的博客" type="application/atom+xml">
</head>


<body>
  

  <header>
    

<div class="header-inner" style="height: 80vh;">
  <nav id="navbar" class="navbar fixed-top  navbar-expand-lg navbar-dark scrolling-navbar">
  <div class="container">
    <a class="navbar-brand" href="/">
      <strong>KD</strong>
    </a>

    <button id="navbar-toggler-btn" class="navbar-toggler" type="button" data-toggle="collapse"
            data-target="#navbarSupportedContent"
            aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
      <div class="animated-icon"><span></span><span></span><span></span></div>
    </button>

    <!-- Collapsible content -->
    <div class="collapse navbar-collapse" id="navbarSupportedContent">
      <ul class="navbar-nav ml-auto text-center">
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/">
                <i class="iconfont icon-home-fill"></i>
                首页
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/archives/">
                <i class="iconfont icon-archive-fill"></i>
                归档
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/categories/">
                <i class="iconfont icon-category-fill"></i>
                分类
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/tags/">
                <i class="iconfont icon-tags-fill"></i>
                标签
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/links/">
                <i class="iconfont icon-link-fill"></i>
                友链
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/atom.xml">
                <i class="iconfont icon-rss"></i>
                RSS
              </a>
            </li>
          
        
        
          <li class="nav-item" id="search-btn">
            <a class="nav-link" target="_self" href="javascript:;" data-toggle="modal" data-target="#modalSearch" aria-label="Search">
              &nbsp;<i class="iconfont icon-search"></i>&nbsp;
            </a>
          </li>
          
        
        
          <li class="nav-item" id="color-toggle-btn">
            <a class="nav-link" target="_self" href="javascript:;" aria-label="Color Toggle">&nbsp;<i
                class="iconfont icon-dark" id="color-toggle-icon"></i>&nbsp;</a>
          </li>
        
      </ul>
    </div>
  </div>
</nav>

  

<div id="banner" class="banner" parallax=true
     style="background: url('https://api.btstu.cn/sjbz/api.php?lx=dongman&format=images') no-repeat center center; background-size: cover;">
  <div class="full-bg-img">
    <div class="mask flex-center" style="background-color: rgba(0, 0, 0, 0.5)">
      <div class="banner-text text-center fade-in-up">
        <div class="h2">
          
            <span id="subtitle" data-typed-text="MOOC爬虫"></span>
          
        </div>

        
          
  <div class="mt-3">
    
    
      <span class="post-meta">
        <i class="iconfont icon-date-fill" aria-hidden="true"></i>
        <time datetime="2021-04-22 09:12" pubdate>
          2021年4月22日 09:12 上午
        </time>
      </span>
    
  </div>

  <div class="mt-1">
    
      <span class="post-meta mr-2">
        <i class="iconfont icon-chart"></i>
        
          5.1k 字
        
      </span>
    

    
      <span class="post-meta mr-2">
        <i class="iconfont icon-clock-fill"></i>
        
        
        
          43 分钟
        
      </span>
    

    
    
      
        <span id="busuanzi_container_page_pv" style="display: none">
          <i class="iconfont icon-eye" aria-hidden="true"></i>
          <span id="busuanzi_value_page_pv"></span> 次
        </span>
        
      
    
  </div>


        
      </div>

      
        <div class="scroll-down-bar">
          <i class="iconfont icon-arrowdown"></i>
        </div>
      
    </div>
  </div>
</div>

</div>

  </header>

  <main>
    
      

<div class="container-fluid nopadding-x">
  <div class="row nomargin-x">
    <div class="side-col d-none d-lg-block col-lg-2">
      

    </div>

    <div class="col-lg-8 nopadding-x-md">
      <div class="container nopadding-x-md" id="board-ctn">
        <div id="board">
          <article class="post-content mx-auto">
            <!-- SEO header -->
            <h1 style="display: none">MOOC爬虫</h1>
            
              <p class="note note-info">
                
                  
                    本文最后更新于：2022年5月18日 晚上
                  
                
              </p>
            
            <div class="markdown-body">
              
              <h2 id="requests模块"><a href="#requests模块" class="headerlink" title="requests模块"></a>requests模块</h2><p><strong>Response对象的属性</strong></p>
<div class="table-container">
<table>
<thead>
<tr>
<th>属性</th>
<th>说明</th>
</tr>
</thead>
<tbody>
<tr>
<td>r.status_code</td>
<td>HTTP请求的返回状态，200表示连接成功，404表示失败</td>
</tr>
<tr>
<td>r.text</td>
<td>HTTP响应内容的字符串形式，即url对应的页面内容</td>
</tr>
<tr>
<td>r.encoding</td>
<td>从HTTP header中猜测的响应内容编码方式</td>
</tr>
<tr>
<td>r.apparent_encoding</td>
<td>从内容中分析出的响应编码方式（备选编码方式）</td>
</tr>
<tr>
<td>r.content</td>
<td>HTTP相应内容的二进制形式</td>
</tr>
</tbody>
</table>
</div>
<p><em>注意</em><br>r.encoding: 如果header中不存在charset，则认为编码为ISO-8859-1<br>r.text根据r.encoding显示网页内容<br>r.apparent_encoding: 根据网页内容分析出的编码方式<br>可以看作是r.encoding的备选</p>
<p><strong>Requests库的异常</strong></p>
<div class="table-container">
<table>
<thead>
<tr>
<th>异常</th>
<th>说明</th>
</tr>
</thead>
<tbody>
<tr>
<td>requests.ConnectionError</td>
<td>网络连接错误异常，如DNS查询失败、拒接俩捏等</td>
</tr>
<tr>
<td>requests.HTTPError</td>
<td>HTTP错误异常</td>
</tr>
<tr>
<td>requests.URLRequired</td>
<td>URL缺失异常</td>
</tr>
<tr>
<td>requests.TooManyRedirects</td>
<td>超过最大重定向次数，产生重定向异常</td>
</tr>
<tr>
<td>requests.ConnectTimeout</td>
<td>连接远程服务器超时异常</td>
</tr>
<tr>
<td>requests.Timeout</td>
<td>请求URL超时，产生超时异常</td>
</tr>
<tr>
<td>requests.raise_for_status()</td>
<td>如果不是200，产生异常 requests.HTTPError</td>
</tr>
</tbody>
</table>
</div>
<p><strong>Requests库的7个重要方法</strong></p>
<div class="table-container">
<table>
<thead>
<tr>
<th>方法</th>
<th>说明</th>
</tr>
</thead>
<tbody>
<tr>
<td>requests.request()</td>
<td>构造一个请求，支撑以下各方法的基础方法</td>
</tr>
<tr>
<td>requests.get()</td>
<td>获取HTML网页的重要方法，对应于HTTP的GET</td>
</tr>
<tr>
<td>requests.head()</td>
<td>获取HTML网页头信息的方法，对应于HTTP的HEAD</td>
</tr>
<tr>
<td>requests.post()</td>
<td>向HTML网页提交POST请求的方法，对应于HTTP的POST</td>
</tr>
<tr>
<td>requests.put()</td>
<td>向HTML网页提交PUT请求的方法，对应于HTTP的PUT</td>
</tr>
<tr>
<td>requests.patch()</td>
<td>向HTML网页提交局部修改请求，对应于HTTP的PATCH</td>
</tr>
<tr>
<td>requests.delete()</td>
<td>向HTML页面提交删除请求，duiyingyuHTTP的DELETE</td>
</tr>
</tbody>
</table>
</div>
<p><strong>HTTP协议</strong></p>
<p><font color = 'red'>HTTP</font>，Hypertext Transfer Protocol，超文本传输协议<br>HTTP是一个基于“请求与响应”模式的，无状态的应用层协议<br>HTTP协议采用URL作为定位网络资源的标识，URL格式如下：</p>
<p><a href="http://host[:port][path">http://host[:port][path</a>]</p>
<p><font color='red'>host</font>: 合法的Internet主机域名或IP地址</p>
<p><font color='red'>port</font>: 端口号，缺省端口为80</p>
<p><font color='red'>path</font>: 请求资源的路径</p>
<p>HTTP URL的理解：<br>URL是通过HTTP协议存取资源的Internet路径，一个URL对应一个数据资源</p>
<p><strong>HTTP协议对资源的操作</strong></p>
<div class="table-container">
<table>
<thead>
<tr>
<th>方法</th>
<th>说明</th>
</tr>
</thead>
<tbody>
<tr>
<td>GET</td>
<td>请求获取URL位置的资源</td>
</tr>
<tr>
<td>HEAD</td>
<td>请求获取URL位置资源的响应消息报告，即获取该资源的头部信息</td>
</tr>
<tr>
<td>POST</td>
<td>请求向URL位置的资源后附加新的数据</td>
</tr>
<tr>
<td>PUT</td>
<td>请求向URL位置存储一个资源，覆盖原URL位置的资源</td>
</tr>
<tr>
<td>PATCH</td>
<td>请求局部更新URL位置的资源，即改变该处资源的部分内容</td>
</tr>
<tr>
<td>DELETE</td>
<td>请求删除URL位置存储的资源</td>
</tr>
</tbody>
</table>
</div>
<p><strong>HTTP协议与Requests库</strong><br>协议方法和Requests库对应的方法功能一致</p>
<p>requests.request(method, url, **kwargs)</p>
<p>method: 请求方式<br>get、head、post、put、patch、delete、optios</p>
<p>**kwargs: 控制访问的参数，均为可选项（13个）</p>
<p><font color='red'>params</font>: 字典或字节序列，作为参数增加到url中<br>r.url: 可以查看请求连接和所带的参数</p>
<p><font color='red'>data</font>: 字典、字节序列或文件对象，作为Request的内容</p>
<p><font color='red'>json</font>: JSON格式的数据，作为Request的内容</p>
<p><font color='red'>headers</font>: 字典，HTTP定制头</p>
<p><font color='red'>cookies</font>: 字典或CookieJar，Request中的cookie</p>
<p><font color='red'>auth</font>: 元组，支持HTTP认证功能</p>
<p><font color='red'>files</font>: 字典类型，传输文件<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><code class="hljs python">fs = &#123;<span class="hljs-string">&#x27;file&#x27;</span>: <span class="hljs-built_in">open</span>(<span class="hljs-string">&#x27;data.xls&#x27;</span>, <span class="hljs-string">&#x27;rb&#x27;</span>)&#125;<br>r = requests.request(<span class="hljs-string">&#x27;POST&#x27;</span>, url, file=fs)<br></code></pre></td></tr></table></figure></p>
<p><font color='red'>timeout</font>: 设定超时时间，单位为秒（超时会产生一个Timeout异常）</p>
<p><font color='red'>proxies</font>: 字典类型，设定访问代理服务器，可以增加登录认证<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><code class="hljs python">pxs = &#123;<span class="hljs-string">&#x27;http&#x27;</span>: <span class="hljs-string">&#x27;http://user:pass@10.10.10.1:1234&#x27;</span>,<br>        <span class="hljs-string">&#x27;https&#x27;</span>: <span class="hljs-string">&#x27;https://10.10.10.1:4321&#x27;</span>&#125;<br>r = requests.request(<span class="hljs-string">&#x27;GET&#x27;</span>, url, proxies=pxs)<br></code></pre></td></tr></table></figure></p>
<p><font color='red'>allow_redirects</font>: True/False，默认为True，<a target="_blank" rel="noopener" href="https://baike.baidu.com/item/重定向">重定向</a>开关</p>
<p><font color='red'>stream</font>: True/False，默认为True，获取内容立即下载开关</p>
<p><font color='red'>verify</font>: True/False，默认为True，认证<a target="_blank" rel="noopener" href="https://baike.baidu.com/item/SSL证书">SSL证书</a>开关</p>
<p><font color='red'>cert</font>: 本地SSL证书路径</p>
<h2 id="Beautiful-Soup库的使用"><a href="#Beautiful-Soup库的使用" class="headerlink" title="Beautiful Soup库的使用"></a>Beautiful Soup库的使用</h2><p>库的小测<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><code class="hljs python"><span class="hljs-keyword">import</span> requests<br><span class="hljs-keyword">from</span> bs4 <span class="hljs-keyword">import</span> BeautifulSoup<br>url = <span class="hljs-string">&quot;http://python123.io/ws/demo.html&quot;</span><br>r = requests.get(url)<br>demo = r.text<br>soup = BeautifulSoup(demo, <span class="hljs-string">&quot;html.parser&quot;</span>) <span class="hljs-comment"># html的解析器</span><br><span class="hljs-built_in">print</span>(soup.prettify())<br></code></pre></td></tr></table></figure></p>
<p>Beautiful Soup库的理解<br>\<p class="title">...\</p><br>\<p>..\</p>: 标签Tag<br>p: 名称Name成对出现<br>属性域Attributes 0个或多个（来定义标签的特点的）</p>
<p><strong>Beautiful Soup库解析器</strong></p>
<div class="table-container">
<table>
<thead>
<tr>
<th>解析器</th>
<th>使用方法</th>
<th>条件</th>
</tr>
</thead>
<tbody>
<tr>
<td>bs4的HTML解析器</td>
<td>BeautifulSoup(mk, ‘html.parser’)</td>
<td>安装bs4库</td>
</tr>
<tr>
<td>lxml的HTML解析器</td>
<td>BeautifulSoup(mk, ‘lxml’)</td>
<td>pip install lxml</td>
</tr>
<tr>
<td>lxml的XML解析器</td>
<td>BeautifulSoup(mk, ‘xml’)</td>
<td>pip install lxml</td>
</tr>
<tr>
<td>html5lib的解析器</td>
<td>BeautifulSoup(mk, ‘html5lib’)</td>
<td>pip install html5lib</td>
</tr>
</tbody>
</table>
</div>
<p><strong>Beautiful Soup类的基本元素</strong></p>
<div class="table-container">
<table>
<thead>
<tr>
<th>基本元素</th>
<th>说明</th>
</tr>
</thead>
<tbody>
<tr>
<td>Tag</td>
<td>标签，最基本的信息组织单元，分别用<>和</>表明开头和结尾</td>
</tr>
<tr>
<td>Name</td>
<td>标签的名字，\<p>...\</p>的名字是’p’，格式：<tag>.name</td>
</tr>
<tr>
<td>Attributes</td>
<td>标签的属性，字典形式组织，格式：<tag>.attrs</td>
</tr>
<tr>
<td>NavigableString</td>
<td>标签内非属性字符串，<>...</>中字符串，格式：<tag>.string</td>
</tr>
<tr>
<td>Comment</td>
<td>标签内字符串的注释部分，一种特殊的Comment类型</td>
</tr>
</tbody>
</table>
</div>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><code class="hljs python"><span class="hljs-built_in">print</span>(soup.title)<br><br><span class="hljs-built_in">print</span>(soup.a.parent.name)<br><br><span class="hljs-built_in">print</span>(soup.a.attrs)<br><span class="hljs-built_in">print</span>(soup.a.attrs[<span class="hljs-string">&#x27;href&#x27;</span>])<br><br>newsoup = BeautifulSoup(<span class="hljs-string">&quot;&lt;b&gt;&lt;!--这是注释--&gt;&lt;/b&gt;&lt;p&gt;这不是注释&lt;/p&gt;&quot;</span>, <span class="hljs-string">&quot;html.parser&quot;</span>)<br><span class="hljs-built_in">print</span>(newsoup.b.string, <span class="hljs-built_in">type</span>(newsoup.b.string))<br><span class="hljs-built_in">print</span>(newsoup.p.sting, <span class="hljs-built_in">type</span>(newsoup.p.string))<br></code></pre></td></tr></table></figure>
<p>bs4库的HTML内容遍历方法<br><strong>标签树的下行遍历</strong></p>
<div class="table-container">
<table>
<thead>
<tr>
<th>属性</th>
<th>说明</th>
</tr>
</thead>
<tbody>
<tr>
<td>.contents</td>
<td>子节点的列表，将<tag>所有儿子节点存入列表</td>
</tr>
<tr>
<td>.children</td>
<td>子节点的迭代类型，与.contents类似，用于循环遍历儿子节点</td>
</tr>
<tr>
<td>.descendants</td>
<td>子孙节点的迭代类型，包含所有子孙节点，用于循环遍历</td>
</tr>
</tbody>
</table>
</div>
<p>遍历<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><code class="hljs python"><span class="hljs-keyword">for</span> child <span class="hljs-keyword">in</span> soup.body.children:<br>        <span class="hljs-built_in">print</span>(child)<br><br><span class="hljs-keyword">for</span> child <span class="hljs-keyword">in</span> soup.body.descendants:<br>        <span class="hljs-built_in">print</span>(child)<br></code></pre></td></tr></table></figure></p>
<p><strong>标签树的上行遍历</strong></p>
<div class="table-container">
<table>
<thead>
<tr>
<th>属性</th>
<th>说明</th>
</tr>
</thead>
<tbody>
<tr>
<td>.parent</td>
<td>节点的父亲标签</td>
</tr>
<tr>
<td>.parents</td>
<td>节点先辈标签的迭代类型，用于循环遍历先辈节点</td>
</tr>
</tbody>
</table>
</div>
<p><strong>标签树的平行遍历</strong></p>
<div class="table-container">
<table>
<thead>
<tr>
<th>属性</th>
<th>说明</th>
</tr>
</thead>
<tbody>
<tr>
<td>.next_sibling</td>
<td>返回按照HTML文本顺序的下一个平行节点标签</td>
</tr>
<tr>
<td>.previous_sibling</td>
<td>返回按照HTML文本顺序的上一个平行节点标签</td>
</tr>
<tr>
<td>.next_siblings</td>
<td>迭代类型，返回按照HTML文本顺序的后续所有平行节点标签</td>
</tr>
<tr>
<td>.previous_siblings</td>
<td>迭代类型，返回按照HTML文本顺序的前续所有平行节点标签</td>
</tr>
</tbody>
</table>
</div>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><code class="hljs python"><span class="hljs-keyword">for</span> sibling <span class="hljs-keyword">in</span> soup.a.next_siblings:<br>        <span class="hljs-built_in">print</span>(sibling)<br><br><span class="hljs-keyword">for</span> sibling <span class="hljs-keyword">in</span> soup.a.previous_siblings:<br>        <span class="hljs-built_in">print</span>(sibling)<br></code></pre></td></tr></table></figure>
<p>实例：提取html文件中所有a标签中的URL连接<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><code class="hljs python">soup = BeautifulSoup(demo, <span class="hljs-string">&quot;html.parser&quot;</span>) <span class="hljs-comment"># html的解析器</span><br><span class="hljs-keyword">for</span> link <span class="hljs-keyword">in</span> soup.findAll(<span class="hljs-string">&#x27;a&#x27;</span>):<br>    <span class="hljs-built_in">print</span>(link.attrs[<span class="hljs-string">&#x27;href&#x27;</span>])<br></code></pre></td></tr></table></figure></p>
<p><strong>基于bs4库的HTML内容查找方法</strong><br>&lt;&gt;.find_all(name, attrs, recursive, string, **kwargs)<br>返回一个列表类型，存储查找的结果。</p>
<p><font color='red'>name</font>: 对标签名称的索引字符串。<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><code class="hljs python">soup.find_all(<span class="hljs-string">&#x27;a&#x27;</span>) <span class="hljs-comment"># 查找a标签</span><br>soup.find_all([<span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>]) <span class="hljs-comment"># 查找a和b标签</span><br>soup.find_all(<span class="hljs-literal">True</span>) <span class="hljs-comment">#所有标签</span><br>suop.find_all(re.<span class="hljs-built_in">compile</span>(<span class="hljs-string">&#x27;b&#x27;</span>)) <span class="hljs-comment">#查找所有以b开头的标签</span><br></code></pre></td></tr></table></figure></p>
<p><font color='red'>attrs</font>: 对<strong>标签属性值</strong>的索引字符串，可标注属性索引。</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><code class="hljs python">soup.find_all(<span class="hljs-string">&#x27;p&#x27;</span>, <span class="hljs-string">&#x27;course&#x27;</span>)<br>soup.find_all(<span class="hljs-built_in">id</span>=<span class="hljs-string">&#x27;link1&#x27;</span>)<br>soup.find_all(<span class="hljs-built_in">id</span>=re.<span class="hljs-built_in">compile</span>(<span class="hljs-string">&#x27;link&#x27;</span>)) <span class="hljs-comment">#查找link开头的标签</span><br></code></pre></td></tr></table></figure>
<p><font color='red'>recursive</font>: 是否对子孙全部索引，默认True。</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><code class="hljs python">soup.find_all(<span class="hljs-string">&#x27;a&#x27;</span>) <span class="hljs-comment">#是有返回值的</span><br>soup.find_all(<span class="hljs-string">&#x27;a&#x27;</span>, recursive=<span class="hljs-literal">False</span>) <span class="hljs-comment">#没有返回值（说明soup的根节点下没有a标签）</span><br></code></pre></td></tr></table></figure>
<p><font color='red'>string</font>: <>...</>中字符串区域的检索字符串。</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><code class="hljs python">soup.find_all(string=<span class="hljs-string">&#x27;Besic Python&#x27;</span>) <span class="hljs-comment">#返回值只有一个</span><br>soup.find_all(string=re.comile(<span class="hljs-string">&#x27;python&#x27;</span>)) <span class="hljs-comment">#返回值不止一个</span><br></code></pre></td></tr></table></figure>
<p>\<tag>(..) 等价于 \<tag>.find_all(..)<br>soup(..) 等价于 soup.find_all(..)</p>
<p><strong>扩展方法</strong></p>
<div class="table-container">
<table>
<thead>
<tr>
<th>方法</th>
<th>说明</th>
</tr>
</thead>
<tbody>
<tr>
<td>&lt;&gt;.find()</td>
<td>搜索且只返回一个结果，字符串类型，同.find_all()参数</td>
</tr>
<tr>
<td>&lt;&gt;.find_parents()</td>
<td>在先辈节点中搜索，返回列表类型，同.find_all()参数</td>
</tr>
<tr>
<td>&lt;&gt;.find_parent()</td>
<td>在先辈节点中返回一个结果，字符串类型，同.find()参数</td>
</tr>
<tr>
<td>&lt;&gt;.find_next_siblings()</td>
<td>在后续平行节点中搜索，返回列表类型，同.find_all参数</td>
</tr>
<tr>
<td>&lt;&gt;.find_next_sibling()</td>
<td>在后续平行节点中返回一个结果，字符串类型，同.find()参数</td>
</tr>
<tr>
<td>&lt;&gt;.find_previous_siblings()</td>
<td>在前序平行节点中搜索，返回列表类型，同.find_all()参数</td>
</tr>
<tr>
<td>&lt;&gt;.find_previous_sibling()</td>
<td>在前序平行节点中返回一个结果，字符串类型，同.find()参数</td>
</tr>
</tbody>
</table>
</div>

              
            </div>
            <hr/>
            <div>
              <div class="post-metas my-3">
  
    <div class="post-meta mr-3 d-flex align-items-center">
      <i class="iconfont icon-category"></i>
      

<span class="category-chains">
  
  
    
      <span class="category-chain">
        
  <a href="/categories/%E7%BC%96%E7%A8%8B/" class="category-chain-item">编程</a>
  
  
    <span>></span>
    
  <a href="/categories/%E7%BC%96%E7%A8%8B/Python/" class="category-chain-item">Python</a>
  
  

  

      </span>
    
  
</span>

    </div>
  
  
    <div class="post-meta">
      <i class="iconfont icon-tags"></i>
      
        <a href="/tags/Python/">#Python</a>
      
    </div>
  
</div>


              
  

  <div class="license-box my-3">
    <div class="license-title">
      <div>MOOC爬虫</div>
      <div>https://kdlong.gitee.io/posts/2021/04-227548fa7ce917.html</div>
    </div>
    <div class="license-meta">
      
        <div class="license-meta-item">
          <div>作者</div>
          <div>KD</div>
        </div>
      
      
        <div class="license-meta-item license-meta-date">
          <div>发布于</div>
          <div>2021年4月22日</div>
        </div>
      
      
      <div class="license-meta-item">
        <div>许可协议</div>
        <div>
          
            
            
              <a target="_blank" href="https://creativecommons.org/licenses/by/4.0/">
              <span class="hint--top hint--rounded" aria-label="BY - 署名">
                <i class="iconfont icon-by"></i>
              </span>
              </a>
            
          
        </div>
      </div>
    </div>
    <div class="license-icon iconfont"></div>
  </div>



              
                <div class="post-prevnext my-3">
                  <article class="post-prev col-6">
                    
                    
                      <a href="/posts/2021/04-22611dbdd61f7e.html" title="Java 导包的使用">
                        <i class="iconfont icon-arrowleft"></i>
                        <span class="hidden-mobile">Java 导包的使用</span>
                        <span class="visible-mobile">上一篇</span>
                      </a>
                    
                  </article>
                  <article class="post-next col-6">
                    
                    
                      <a href="/posts/2021/04-22b180539b358b.html" title="C语言">
                        <span class="hidden-mobile">C语言</span>
                        <span class="visible-mobile">下一篇</span>
                        <i class="iconfont icon-arrowright"></i>
                      </a>
                    
                  </article>
                </div>
              
            </div>

            
          </article>
        </div>
      </div>
    </div>

    <div class="side-col d-none d-lg-block col-lg-2">
      
  <aside class="sidebar" style="margin-left: -1rem">
    <div id="toc">
  <p class="toc-header"><i class="iconfont icon-list"></i>&nbsp;目录</p>
  <div class="toc-body" id="toc-body"></div>
</div>



  </aside>


    </div>
  </div>
</div>





  



  



  



  



  


  
  








    

    
      <a id="scroll-top-button" aria-label="TOP" href="#" role="button">
        <i class="iconfont icon-arrowup" aria-hidden="true"></i>
      </a>
    

    
      <div class="modal fade" id="modalSearch" tabindex="-1" role="dialog" aria-labelledby="ModalLabel"
     aria-hidden="true">
  <div class="modal-dialog modal-dialog-scrollable modal-lg" role="document">
    <div class="modal-content">
      <div class="modal-header text-center">
        <h4 class="modal-title w-100 font-weight-bold">搜索</h4>
        <button type="button" id="local-search-close" class="close" data-dismiss="modal" aria-label="Close">
          <span aria-hidden="true">&times;</span>
        </button>
      </div>
      <div class="modal-body mx-3">
        <div class="md-form mb-5">
          <input type="text" id="local-search-input" class="form-control validate">
          <label data-error="x" data-success="v" for="local-search-input">关键词</label>
        </div>
        <div class="list-group" id="local-search-result"></div>
      </div>
    </div>
  </div>
</div>

    

    
  </main>

  <footer>
    <div class="footer-inner">
  
    <div class="footer-content">
       <!--<a href="https://hexo.io" target="_blank" rel="nofollow noopener"><span>Hexo</span></a>--> <div><i class="iconfont icon-love"></i> <a href="https://gitee.com/kdlong/" target="_blank" rel="nofollow noopener"><span>KD 的Gitee</span></a></div> <div> <span id="timeDate">载入天数...</span> <span id="times">载入时分秒...</span> <script src="/js/duration.js"></script> </div> 
    </div>
  
  
    <div class="statistics">
  
  

  
    
      <span id="busuanzi_container_site_pv" style="display: none">
        总访问量 
        <span id="busuanzi_value_site_pv"></span>
         次
      </span>
    
    
      <span id="busuanzi_container_site_uv" style="display: none">
        总访客数 
        <span id="busuanzi_value_site_uv"></span>
         人
      </span>
    
    
  
</div>

  
  
  
</div>

  </footer>

  <!-- Scripts -->
  
  <script  src="https://lib.baomitu.com/nprogress/0.2.0/nprogress.min.js" ></script>
  <link  rel="stylesheet" href="https://lib.baomitu.com/nprogress/0.2.0/nprogress.min.css" />

  <script>
    NProgress.configure({"showSpinner":false,"trickleSpeed":100})
    NProgress.start()
    window.addEventListener('load', function() {
      NProgress.done();
    })
  </script>


<script  src="https://lib.baomitu.com/jquery/3.6.0/jquery.min.js" ></script>
<script  src="https://lib.baomitu.com/twitter-bootstrap/4.6.1/js/bootstrap.min.js" ></script>
<script  src="/js/events.js" ></script>
<script  src="/js/plugins.js" ></script>


  <script  src="https://lib.baomitu.com/typed.js/2.0.12/typed.min.js" ></script>
  <script>
    (function (window, document) {
      var typing = Fluid.plugins.typing;
      var subtitle = document.getElementById('subtitle');
      if (!subtitle || !typing) {
        return;
      }
      var text = subtitle.getAttribute('data-typed-text');
      
        typing(text);
      
    })(window, document);
  </script>






  
    <script  src="/js/img-lazyload.js" ></script>
  




  
<script>
  Fluid.utils.createScript('https://lib.baomitu.com/tocbot/4.18.0/tocbot.min.js', function() {
    var toc = jQuery('#toc');
    if (toc.length === 0 || !window.tocbot) { return; }
    var boardCtn = jQuery('#board-ctn');
    var boardTop = boardCtn.offset().top;

    window.tocbot.init({
      tocSelector     : '#toc-body',
      contentSelector : '.markdown-body',
      headingSelector : CONFIG.toc.headingSelector || 'h1,h2,h3,h4,h5,h6',
      linkClass       : 'tocbot-link',
      activeLinkClass : 'tocbot-active-link',
      listClass       : 'tocbot-list',
      isCollapsedClass: 'tocbot-is-collapsed',
      collapsibleClass: 'tocbot-is-collapsible',
      collapseDepth   : CONFIG.toc.collapseDepth || 0,
      scrollSmooth    : true,
      headingsOffset  : -boardTop
    });
    if (toc.find('.toc-list-item').length > 0) {
      toc.css('visibility', 'visible');
    }
  });
</script>


  <script>
  (function() {
    var enableLang = CONFIG.code_language.enable && CONFIG.code_language.default;
    var enableCopy = CONFIG.copy_btn;
    if (!enableLang && !enableCopy) {
      return;
    }

    function getBgClass(ele) {
      return Fluid.utils.getBackgroundLightness(ele) >= 0 ? 'code-widget-light' : 'code-widget-dark';
    }

    var copyTmpl = '';
    copyTmpl += '<div class="code-widget">';
    copyTmpl += 'LANG';
    copyTmpl += '</div>';
    jQuery('.markdown-body pre').each(function() {
      var $pre = jQuery(this);
      if ($pre.find('code.mermaid').length > 0) {
        return;
      }
      if ($pre.find('span.line').length > 0) {
        return;
      }

      var lang = '';

      if (enableLang) {
        lang = CONFIG.code_language.default;
        if ($pre[0].children.length > 0 && $pre[0].children[0].classList.length >= 2 && $pre.children().hasClass('hljs')) {
          lang = $pre[0].children[0].classList[1];
        } else if ($pre[0].getAttribute('data-language')) {
          lang = $pre[0].getAttribute('data-language');
        } else if ($pre.parent().hasClass('sourceCode') && $pre[0].children.length > 0 && $pre[0].children[0].classList.length >= 2) {
          lang = $pre[0].children[0].classList[1];
          $pre.parent().addClass('code-wrapper');
        } else if ($pre.parent().hasClass('markdown-body') && $pre[0].classList.length === 0) {
          $pre.wrap('<div class="code-wrapper"></div>');
        }
        lang = lang.toUpperCase().replace('NONE', CONFIG.code_language.default);
      }
      $pre.append(copyTmpl.replace('LANG', lang).replace('code-widget">',
        getBgClass($pre[0]) + (enableCopy ? ' code-widget copy-btn" data-clipboard-snippet><i class="iconfont icon-copy"></i>' : ' code-widget">')));

      if (enableCopy) {
        Fluid.utils.createScript('https://lib.baomitu.com/clipboard.js/2.0.10/clipboard.min.js', function() {
          var clipboard = new window.ClipboardJS('.copy-btn', {
            target: function(trigger) {
              var nodes = trigger.parentNode.childNodes;
              for (var i = 0; i < nodes.length; i++) {
                if (nodes[i].tagName === 'CODE') {
                  return nodes[i];
                }
              }
            }
          });
          clipboard.on('success', function(e) {
            e.clearSelection();
            e.trigger.innerHTML = e.trigger.innerHTML.replace('icon-copy', 'icon-success');
            setTimeout(function() {
              e.trigger.innerHTML = e.trigger.innerHTML.replace('icon-success', 'icon-copy');
            }, 2000);
          });
        });
      }
    });
  })();
</script>


  
<script>
  Fluid.utils.createScript('https://lib.baomitu.com/anchor-js/4.3.1/anchor.min.js', function() {
    window.anchors.options = {
      placement: CONFIG.anchorjs.placement,
      visible  : CONFIG.anchorjs.visible
    };
    if (CONFIG.anchorjs.icon) {
      window.anchors.options.icon = CONFIG.anchorjs.icon;
    }
    var el = (CONFIG.anchorjs.element || 'h1,h2,h3,h4,h5,h6').split(',');
    var res = [];
    for (var item of el) {
      res.push('.markdown-body > ' + item.trim());
    }
    if (CONFIG.anchorjs.placement === 'left') {
      window.anchors.options.class = 'anchorjs-link-left';
    }
    window.anchors.add(res.join(', '));
  });
</script>


  
<script>
  Fluid.utils.createScript('https://lib.baomitu.com/fancybox/3.5.7/jquery.fancybox.min.js', function() {
    Fluid.plugins.fancyBox();
  });
</script>


  <script>Fluid.plugins.imageCaption();</script>

  
      <script>
        MathJax = {
          tex    : {
            inlineMath: { '[+]': [['$', '$']] }
          },
          loader : {
            load: ['ui/lazy']
          },
          options: {
            renderActions: {
              findScript    : [10, doc => {
                document.querySelectorAll('script[type^="math/tex"]').forEach(node => {
                  const display = !!node.type.match(/; *mode=display/);
                  const math = new doc.options.MathItem(node.textContent, doc.inputJax[0], display);
                  const text = document.createTextNode('');
                  node.parentNode.replaceChild(text, node);
                  math.start = { node: text, delim: '', n: 0 };
                  math.end = { node: text, delim: '', n: 0 };
                  doc.math.push(math);
                });
              }, '', false],
              insertedScript: [200, () => {
                document.querySelectorAll('mjx-container').forEach(node => {
                  let target = node.parentNode;
                  if (target.nodeName.toLowerCase() === 'li') {
                    target.parentNode.classList.add('has-jax');
                  }
                });
              }, '', false]
            }
          }
        };
      </script>
    

  <script  src="https://lib.baomitu.com/mathjax/3.2.0/es5/tex-mml-chtml.js" ></script>

  <script  src="/js/local-search.js" ></script>

  <script defer src="https://busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js" ></script>





<!-- 主题的启动项，将它保持在最底部 -->
<!-- the boot of the theme, keep it at the bottom -->
<script  src="/js/boot.js" ></script>


  

  <noscript>
    <div class="noscript-warning">博客在允许 JavaScript 运行的环境下浏览效果更佳</div>
  </noscript>
</body>
</html>
